package poradaagreementsparser;

import java.io.*;
import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


import org.jdom.Attribute;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import org.jdom.output.XMLOutputter;
import org.jdom.xpath.XPath;



/**
 * @author Martin Mihalovic
 */
public class PoradaAgreementsParser {

    private static String sourceDir;
    private static File target;
    private FileLoader fileLoader;
    private BufferedWriter writer;
    
    /**
     * @param args source directory with discussion posts (HTML files) and target text file
     *             where parsed useful/useless ratings will be saved.
     */
    public static void main(String[] args) throws IOException, FileNotFoundException, JDOMException {

        if(args.length == 2){
            sourceDir = args[0];
            if(!sourceDir.endsWith("\\"))
                sourceDir += "\\";
            File dir = new File(sourceDir);
            if(!dir.exists() || !dir.isDirectory()){
                System.err.println("Invalid input directory " + sourceDir);
                return;
            }

            target = new File(args[1]);
            if(!target.createNewFile()){
                System.out.println("File " + args[1] + " already exists. Its content will be deleted");
                new FileOutputStream(target).write('\n');       //Note: stream not closed
            }
        }
        else {
            System.err.println("Usage: parser <folder_with_HTML_posts> <output_file>");
            return;
        }
        
        PoradaAgreementsParser parser = new PoradaAgreementsParser();
        parser.parse();
        
        System.out.println("FINISHED. Press ENTER to quit");
        System.in.read();
    }

    
    public PoradaAgreementsParser() throws IOException {
        log("\n =============== User ratings parser began at " + new Date() + " =============== \n");
        fileLoader = new FileLoader(sourceDir);
        writer = new BufferedWriter(new FileWriter(target));
        writer.write("INSERT INTO rating(fromUserID, toUserID, domainID, weight) VALUES");
        writer.newLine();
    }


    private void parse() throws FileNotFoundException, IOException, JDOMException {
        
        Pattern domainPattern = Pattern.compile("<li class=\"navbit\">\\s*<a[^>]*href=\"http://www.porada.sk/(f\\d+)-[\\w-]+.html\"[^>]*>[^<]+</a>\\s*</li>\\s*<li class=\"navbit lastnavbit\">");
        Pattern usefulPattern = Pattern.compile("([^>]+)<br\\s*clear=['\"]none['\"]\\s*/>");           //Also used for 'useless'
        Pattern userIdPattern = Pattern.compile("member\\.php\\?u=(\\d+)");
        XMLOutputter outputer = new XMLOutputter();
        
        XPath domainPath = XPath.newInstance("//h:div[@id='breadcrumb']/h:ul[@class='floatcontainer']");        
        XPath postPath = XPath.newInstance("//h:li[@class='postbitlegacy postbitim postcontainer']");
        XPath autorIdPath = XPath.newInstance("./h:div[@class='postdetails']/h:div[@class='userinfo']/h:div[@class='username_container']/h:div/h:a[@rel='nofollow']/@href");
        XPath miniPostAuthorIdPath = XPath.newInstance("./h:table/h:tr/h:td/h:span/h:a[@rel='nofollow']/@href");
        XPath miniPostAuthorIdPath2 = XPath.newInstance("./h:table/h:tr/h:td[@class='alt1']/h:a[@rel='nofollow']/@href");
        XPath ratingsPath = XPath.newInstance("./h:div[@class='postfoot']/h:div/h:div[@class='ratingdiv']/h:a[@class='bubble hiderating' or @class='bubble hiderating postbitalpha']");
        XPath miniRatingsPath = XPath.newInstance("./h:table/h:tr/h:td[@class='alt1']/h:span[@class='smallfont']/h:a[@class='bubble hiderating']");
        XPath usefulPath = XPath.newInstance("./h:span[@class='bubble']");      //Used for both 'useful' and 'useless'
        domainPath.addNamespace("h", "http://www.w3.org/1999/xhtml");
        postPath.addNamespace("h", "http://www.w3.org/1999/xhtml");
        autorIdPath.addNamespace("h", "http://www.w3.org/1999/xhtml");
        miniPostAuthorIdPath.addNamespace("h", "http://www.w3.org/1999/xhtml");
        miniPostAuthorIdPath2.addNamespace("h", "http://www.w3.org/1999/xhtml");
        ratingsPath.addNamespace("h", "http://www.w3.org/1999/xhtml");
        miniRatingsPath.addNamespace("h", "http://www.w3.org/1999/xhtml");
        usefulPath.addNamespace("h", "http://www.w3.org/1999/xhtml");
        
        String fileName;
        while( (fileName = fileLoader.GetNextFile()) != null) {
            
            try {
                int usefulCount = 0;
                int uselessCount = 0;

                System.out.println("\n\nParsing file " + fileName);

                File file = new File(fileName);
                SAXBuilder builder = new SAXBuilder("org.ccil.cowan.tagsoup.Parser");
                Document doc = builder.build(file);

                Element bcDiv = (Element)domainPath.selectSingleNode(doc);
                if(bcDiv == null) {
                    log(fileName + ": No breadcrumb");
                    continue;
                }

                String bcText = outputer.outputString(bcDiv);

                String domId = "<ERROR :'(>";

                Matcher m = domainPattern.matcher(bcText);
                if(m.find()) {
                    domId = m.group(1);
    //                System.out.println("DomainID = " + domId);
                }
                else {
                    log(fileName, "ERROR: Unable to identify domain!");
                    continue;
                }

                List posts = postPath.selectNodes(doc);

                for(Object post : posts) {
                    Attribute author = (Attribute) autorIdPath.selectSingleNode(post);
                    XPath _ratingsPath = ratingsPath; 
                    Boolean miniPost = false;

                    int authorId = -1;

                    if(author == null) {      //Expecting mini post
                        miniPost = true;
                        author = (Attribute) miniPostAuthorIdPath.selectSingleNode(post);
                        if(author == null)
                            author = (Attribute) miniPostAuthorIdPath2.selectSingleNode(post);
    //                    System.out.println("Author ID (from link): " + authorId);
                        _ratingsPath = miniRatingsPath;
                    }

                    if(author==null) {
                        log(fileName + " : Cannot parse author for post " + ((Element)post).getAttributeValue("id"));
                        continue;
                    }

                    m = userIdPattern.matcher(author.toString());
                    m.find();
                    authorId = Integer.parseInt(m.group(1));

                    List ratingLinks = _ratingsPath.selectNodes(post);

                    if(ratingLinks.isEmpty()) {         //Totaly incorrect HTML (ex. t100147-p10-dohoda-o-vykonani-prace.html)
                        log(fileName + " : rating link not found for author " + author + " (mini-post: " + miniPost + ")");
                        continue;
                    }

                    Element usefulSpan = (Element) usefulPath.selectSingleNode(ratingLinks.get(0));
                    if(usefulSpan!=null) {       //Else no 'useful' ratings given

                        String spanText = outputer.outputString(usefulSpan);
                        m = usefulPattern.matcher(spanText);
                        while(m.find()) {
                            String userNick = m.group(1);
                            usefulCount++;
                            saveRecord(userNick, authorId, domId, +1.0);
    //                        System.out.println(userNick + " ranked useful post");
                            if(miniPost)
                                log(fileName + " : USEFULL MINI-POST!");
                        }
                    }

                    Element uselessSpan = (Element) usefulPath.selectSingleNode(ratingLinks.get(1));
                    if(uselessSpan!=null) {       //Else no 'useful' ratings given

                        String spanText = outputer.outputString(uselessSpan);
                        m = usefulPattern.matcher(spanText);
                        while(m.find()) {
                            String userNick = m.group(1);
                            uselessCount++;
                            saveRecord(userNick, authorId, domId, -1.0);
    //                        System.out.println(userNick + " ranked useless post");
                            if(miniPost)
                                log(fileName + " : USELESS MINI-POST!");
                        }
                    }
                }

                System.out.println("Ratings saved: " + usefulCount + " useful, " + uselessCount + " useless.");
            }
            catch(Exception e) {            //Unexpected errors
                log(fileName + " : ERROR " + e.getMessage() + "\n" + e.getStackTrace());
                continue;
            }
        }
        
        writer.close();
    }


    int counter = 0;
    
    private void saveRecord(String fromNodeNick, int toNodeId, String domainId, double weight) throws IOException {
        writer.write("((SELECT userID from user where name ='" + fromNodeNick.trim().replace('\'', '`').replace("\\", "\\\\") + "'), " + toNodeId + ", '" + domainId +  "', " + weight + ")");
        counter++;
        if(counter % 150 == 0) {
            writer.write(";");
            writer.newLine();
            writer.write("INSERT INTO rating(fromUserID, toUserID, domainID, weight) VALUES");
            writer.newLine();
        }
        else if (counter % 5 == 0) {
            writer.write(", ");
            writer.newLine();
        }
        else writer.write(", ");
    }
    
    
    /**
     * Logs a single line into log file placed at root of this application.
     * @param message a text to be logged. Usually error description.
     * @throws IOException 
     */
    private void log(String fileName, String message) throws IOException{
        System.err.println(message);
        
        FileWriter fWriter = new FileWriter("UserRatingsParser.log", true);
        BufferedWriter out = new BufferedWriter(fWriter);
        out.write(fileName + ": " + message);
        out.newLine();
        out.close();
    }
    
    private void log(String message) throws IOException {
        log("", message);
    }
}
